{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c92e10e0",
   "metadata": {},
   "source": [
    "动作的熵加入loss,熵越大越好,这样能让模型更健壮"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91162df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAUH0lEQVR4nO3df2xT97kG8MdObJOQHKcJxG5uYoHUbjTix7YA4VDddVozsjaaxsgf3Ya6rEKgMgeVZuJquWqp6HrliumqW1caJlWD/tMxZbpsakSLckMbVNUQSJcpBEhbld5kgO0CzXGSEtux3/sHy1ldAotJ4q9dPx/pSD3n+9p+zyF+en742BYRERARKWBV3QAR5S4GEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKaMsgPbt24clS5ZgwYIFqK2tRU9Pj6pWiEgRJQH0xz/+ES0tLXjmmWfw3nvvYdWqVaivr0coFFLRDhEpYlFxM2ptbS3WrFmDl156CQCQSCRQVVWFHTt24Be/+EW62yEiRfLT/YLRaBS9vb1obW01l1mtVtTV1cHv90/7mEgkgkgkYs4nEglcu3YNZWVlsFgs894zEaVGRDA6OoqKigpYrbc+0Ep7AF25cgXxeBwulytpucvlwvnz56d9jM/nw549e9LRHhHNoeHhYVRWVt5yPO0BdCdaW1vR0tJizhuGAY/Hg+HhYWiaprAzIppOOBxGVVUViouLb1uX9gBatGgR8vLyEAwGk5YHg0G43e5pH+NwOOBwOG5armkaA4gog/2rUyRpvwpmt9tRU1ODrq4uc1kikUBXVxd0XU93O0SkkJJDsJaWFjQ1NWH16tVYu3Ytfv3rX2N8fByPPfaYinaISBElAfTII4/gk08+we7duxEIBPC1r30Nb7755k0nponoy03J54BmKxwOw+l0wjAMngMiykAzfY/yXjAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpk3IAHT9+HN/73vdQUVEBi8WCP//5z0njIoLdu3fj7rvvRkFBAerq6vDBBx8k1Vy7dg2bN2+GpmkoKSnBli1bMDY2NqsVIaLsk3IAjY+PY9WqVdi3b9+043v37sWLL76I/fv34+TJk1i4cCHq6+sxMTFh1mzevBkDAwPo7OxER0cHjh8/jm3btt35WhBRdpJZACCHDx825xOJhLjdbvnVr35lLhsZGRGHwyF/+MMfRETk7NmzAkBOnTpl1rzxxhtisVjk4sWLM3pdwzAEgBiGMZv2iWiezPQ9OqfngC5cuIBAIIC6ujpzmdPpRG1tLfx+PwDA7/ejpKQEq1evNmvq6upgtVpx8uTJaZ83EokgHA4nTUSU/eY0gAKBAADA5XIlLXe5XOZYIBBAeXl50nh+fj5KS0vNmi/y+XxwOp3mVFVVNZdtE5EiWXEVrLW1FYZhmNPw8LDqlohoDsxpALndbgBAMBhMWh4MBs0xt9uNUCiUND45OYlr166ZNV/kcDigaVrSRETZb04DaOnSpXC73ejq6jKXhcNhnDx5ErquAwB0XcfIyAh6e3vNmmPHjiGRSKC2tnYu2yGiDJef6gPGxsbw4YcfmvMXLlxAX18fSktL4fF4sHPnTjz33HO49957sXTpUjz99NOoqKjAxo0bAQD33Xcfvvvd72Lr1q3Yv38/YrEYmpub8cMf/hAVFRVztmJElAVSvbz21ltvCYCbpqamJhG5cSn+6aefFpfLJQ6HQx588EEZHBxMeo6rV6/Kj370IykqKhJN0+Sxxx6T0dHROb/ER0RqzPQ9ahERUZh/dyQcDsPpdMIwDJ4PIspAM32PZsVVMCL6cmIAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMqk/LM8RLM1/sn/YWLknz/DbVtYguK7vwKLxaKwK1KBAURpd/WDkwj2/685r1VWo/juryjsiFThIRgpJ5IAJKG6DVKAAUTqJRLIwp+noznAACLlJBEHGEA5iQFEyokkbhyGUc5hAJFykkhwDyhHMYBIOZE4BAygXJRSAPl8PqxZswbFxcUoLy/Hxo0bMTg4mFQzMTEBr9eLsrIyFBUVobGxEcFgMKlmaGgIDQ0NKCwsRHl5OXbt2oXJycnZrw1lBavNnjSfiEUgcf7756KUAqi7uxterxcnTpxAZ2cnYrEYNmzYgPHxcbPmySefxOuvv4729nZ0d3fj0qVL2LRpkzkej8fR0NCAaDSKd999F6+++ioOHjyI3bt3z91aUUZbUOIGPvehw9hnBuLR6wo7ImVkFkKhkACQ7u5uEREZGRkRm80m7e3tZs25c+cEgPj9fhEROXLkiFitVgkEAmZNW1ubaJomkUhkRq9rGIYAEMMwZtM+KXL1w1PS87tt0rN/q/Ts3yqnX/HK9ZGg6rZoDs30PTqrc0CGYQAASktLAQC9vb2IxWKoq6sza5YtWwaPxwO/3w8A8Pv9WLFiBVwul1lTX1+PcDiMgYGBaV8nEokgHA4nTZS9LHn8AD7dcMcBlEgksHPnTtx///1Yvnw5ACAQCMBut6OkpCSp1uVyIRAImDWfD5+p8amx6fh8PjidTnOqqqq607YpA1iseQB43xfNIoC8Xi/OnDmDQ4cOzWU/02ptbYVhGOY0PDw8769J88di5R4Q3XBHfwnNzc3o6OjA8ePHUVlZaS53u92IRqMYGRlJ2gsKBoNwu91mTU9PT9LzTV0lm6r5IofDAYfDcSetUgayWvNUt0AZIqU9IBFBc3MzDh8+jGPHjmHp0qVJ4zU1NbDZbOjq6jKXDQ4OYmhoCLquAwB0XUd/fz9CoZBZ09nZCU3TUF1dPZt1oSzBc0A0JaW/BK/Xi9deew1/+ctfUFxcbJ6zcTqdKCgogNPpxJYtW9DS0oLS0lJomoYdO3ZA13WsW7cOALBhwwZUV1fj0Ucfxd69exEIBPDUU0/B6/VyLydHWLgHRP+QUgC1tbUBAL71rW8lLT9w4AB++tOfAgBeeOEFWK1WNDY2IhKJoL6+Hi+//LJZm5eXh46ODmzfvh26rmPhwoVoamrCs88+O7s1oaxhsXxhx1uE94LlKItI9t2EEw6H4XQ6YRgGNE1T3Q6l6LMrwxj4n/8yvwPIkmdD9ab/RGHpvynujObKTN+jvBeMMoLE46pbIAUYQJQBBJLgvWC5iAFE6sk/vpSMcg4DiDICAyg3MYAoA/AQLFcxgCj9LJabfgMsMRlT1AypxACitMtfUARbodOcl0Q86YcKKXcwgCjtLNa8mz4NzXNAuYkBRGlnsVh5OwYBYACRChbrjYlyHv8KKO0s1ptPQlNuYgBR2vEQjKYwgCj9LNakX8Wg3MUAorSzWCw3fyUHbnzhHeUWBhCl3zQBxPDJTQwgygi8FSM3MYAoI/CDiLmJAUQZIRGPA+BhWK5hAFFG4CFYbmIAUUaQRJw7QDmIAUQZ4cYeEBMo1zCAKCPwS+lzEwOIFLDAoS1OWhIZ/YS/DZaDGECkRH5BcdJ8PHId4IcRcw5/pJvmRTwex+jo6C1GBdFo8lWveCIBwzBgybNN+wibzYaFCxfOcZekGgOI5sX777+Phx9+GLHY9N/1/Mg378Ej3/yKOf/xhQvYVFOD6OT0h2EbN27ESy+9NC+9kjoMIJoXsVgMFy9evGUAXf10MQLRpRieuA+FeWHYo124eOkSorHpT0Z/+umn89kuKZLSOaC2tjasXLkSmqZB0zTouo433njDHJ+YmIDX60VZWRmKiorQ2NiIYDCY9BxDQ0NoaGhAYWEhysvLsWvXLkxO8kNouSZw/W70jz2Aq7FKDE/chzNj/w4RnpLMNSn9i1dWVuL5559Hb28vTp8+jW9/+9v4/ve/j4GBAQDAk08+iddffx3t7e3o7u7GpUuXsGnTJvPx8XgcDQ0NiEajePfdd/Hqq6/i4MGD2L1799yuFWW80Wgh4mL/x5wF4/ESCPgdQTlHZumuu+6SV155RUZGRsRms0l7e7s5du7cOQEgfr9fRESOHDkiVqtVAoGAWdPW1iaapkkkEpnxaxqGIQDEMIzZtk/z5G9/+5vYbDbBjU8X3jTVr18n/+07Knue65FfPucX33/4xJ6ff8v6H//4x6pXiVIw0/foHZ8DisfjaG9vx/j4OHRdR29vL2KxGOrq6syaZcuWwePxwO/3Y926dfD7/VixYgVcLpdZU19fj+3bt2NgYABf//rXU+rh/PnzKCoqutNVoHn00Ucf3fY7fj4aeh8nun6JUNSDgrwxOGIfIn6bO+INw8DZs2fno1WaB2NjYzOqSzmA+vv7oes6JiYmUFRUhMOHD6O6uhp9fX2w2+0oKSlJqne5XAgEbvzoXCAQSAqfqfGpsVuJRCKIRCLmfDgcBnDjj5LnjzLTrS/B3/DB36/hg7+/M+Pni0ajGBkZmWVXlC7j4+Mzqks5gL761a+ir68PhmHgT3/6E5qamtDd3Z1yg6nw+XzYs2fPTctra2uhadq8vjbdmaKiojn95YvFixdj/fr1c/Z8NL+mdhL+lZQvO9jtdtxzzz2oqamBz+fDqlWr8Jvf/AZut3va/0sFg0G43W4AgNvtvumq2NT8VM10WltbYRiGOQ0PD6faNhFloFlf90wkEohEIqipqYHNZkNXV5c5Njg4iKGhIei6DgDQdR39/f0IhUJmTWdnJzRNQ3V19S1fw+FwmJf+pyYiyn4pHYK1trbioYcegsfjwejoKF577TW8/fbbOHr0KJxOJ7Zs2YKWlhaUlpZC0zTs2LEDuq5j3bp1AIANGzaguroajz76KPbu3YtAIICnnnoKXq8XDodjXlaQiDJXSgEUCoXwk5/8BJcvX4bT6cTKlStx9OhRfOc73wEAvPDCC7BarWhsbEQkEkF9fT1efvll8/F5eXno6OjA9u3boes6Fi5ciKamJjz77LNzu1akXF5eHjRNu+UnoVNVUFAwJ89DmcUit7tWmqHC4TCcTicMw+DhWIaKRqM3ne+bjcLCQpSVlc3Z89H8mul7lPeC0byw2+2oqqpS3QZlON58Q0TKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJTJV93AnRARAEA4HFbcCRFNZ+q9OfVevZWsDKCrV68CAKqqqhR3QkS3Mzo6CqfTecvxrAyg0tJSAMDQ0NBtV46ShcNhVFVVYXh4GJqmqW4nK3Cb3RkRwejoKCoqKm5bl5UBZLXeOHXldDr5R3EHNE3jdksRt1nqZrJzwJPQRKQMA4iIlMnKAHI4HHjmmWfgcDhUt5JVuN1Sx202vyzyr66TERHNk6zcAyKiLwcGEBEpwwAiImUYQESkTFYG0L59+7BkyRIsWLAAtbW16OnpUd2SMj6fD2vWrEFxcTHKy8uxceNGDA4OJtVMTEzA6/WirKwMRUVFaGxsRDAYTKoZGhpCQ0MDCgsLUV5ejl27dmFycjKdq6LM888/D4vFgp07d5rLuM3SRLLMoUOHxG63y+9//3sZGBiQrVu3SklJiQSDQdWtKVFfXy8HDhyQM2fOSF9fnzz88MPi8XhkbGzMrHn88celqqpKurq65PTp07Ju3TpZv369OT45OSnLly+Xuro6+etf/ypHjhyRRYsWSWtrq4pVSquenh5ZsmSJrFy5Up544glzObdZemRdAK1du1a8Xq85H4/HpaKiQnw+n8KuMkcoFBIA0t3dLSIiIyMjYrPZpL293aw5d+6cABC/3y8iIkeOHBGr1SqBQMCsaWtrE03TJBKJpHcF0mh0dFTuvfde6ezslAceeMAMIG6z9MmqQ7BoNIre3l7U1dWZy6xWK+rq6uD3+xV2ljkMwwDwzxt2e3t7EYvFkrbZsmXL4PF4zG3m9/uxYsUKuFwus6a+vh7hcBgDAwNp7D69vF4vGhoakrYNwG2WTll1M+qVK1cQj8eT/tEBwOVy4fz584q6yhyJRAI7d+7E/fffj+XLlwMAAoEA7HY7SkpKkmpdLhcCgYBZM902nRr7Mjp06BDee+89nDp16qYxbrP0yaoAotvzer04c+YM3nnnHdWtZLTh4WE88cQT6OzsxIIFC1S3k9Oy6hBs0aJFyMvLu+lqRDAYhNvtVtRVZmhubkZHRwfeeustVFZWmsvdbjei0ShGRkaS6j+/zdxu97TbdGrsy6a3txehUAjf+MY3kJ+fj/z8fHR3d+PFF19Efn4+XC4Xt1maZFUA2e121NTUoKury1yWSCTQ1dUFXdcVdqaOiKC5uRmHDx/GsWPHsHTp0qTxmpoa2Gy2pG02ODiIoaEhc5vpuo7+/n6EQiGzprOzE5qmobq6Oj0rkkYPPvgg+vv70dfXZ06rV6/G5s2bzf/mNksT1WfBU3Xo0CFxOBxy8OBBOXv2rGzbtk1KSkqSrkbkku3bt4vT6ZS3335bLl++bE6fffaZWfP444+Lx+ORY8eOyenTp0XXddF13RyfuqS8YcMG6evrkzfffFMWL16cU5eUP38VTITbLF2yLoBERH7729+Kx+MRu90ua9eulRMnTqhuSRkA004HDhwwa65fvy4/+9nP5K677pLCwkL5wQ9+IJcvX056no8//lgeeughKSgokEWLFsnPf/5zicViaV4bdb4YQNxm6cGv4yAiZbLqHBARfbkwgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImX+H9OZxWT094aAAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ecfbe912",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "  (1): ReLU()\n",
       "  (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "  (3): ReLU()\n",
       "  (4): Linear(in_features=64, out_features=2, bias=True)\n",
       "  (5): Softmax(dim=1)\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#定义模型,计算每个动作的概率\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    "    torch.nn.Softmax(dim=1),\n",
    ")\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84cbf0ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\appDir\\python3.10\\lib\\site-packages\\gym\\utils\\passive_env_checker.py:233: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`.  (Deprecated NumPy 1.24)\n",
      "  if not isinstance(terminated, (bool, np.bool8)):\n",
      "C:\\Users\\Administrator\\AppData\\Local\\Temp\\ipykernel_13724\\449353166.py:30: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ..\\torch\\csrc\\utils\\tensor_new.cpp:248.)\n",
      "  state = torch.FloatTensor(state).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-983.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    state = []\n",
    "    action = []\n",
    "    reward = []\n",
    "\n",
    "    s = env.reset()\n",
    "    o = False\n",
    "    while not o:\n",
    "        #根据概率采样\n",
    "        prob = model(torch.FloatTensor(s).reshape(1, 4))[0].tolist()\n",
    "        a = random.choices(range(2), weights=prob, k=1)[0]\n",
    "\n",
    "        ns, r, o = env.step(a)\n",
    "\n",
    "        state.append(s)\n",
    "        action.append(a)\n",
    "        reward.append(r)\n",
    "\n",
    "        s = ns\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    state = torch.FloatTensor(state).reshape(-1, 4)\n",
    "    action = torch.LongTensor(action).reshape(-1, 1)\n",
    "    reward = torch.FloatTensor(reward).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, reward.sum().item()\n",
    "\n",
    "\n",
    "state, action, reward, reward_sum = play()\n",
    "\n",
    "reward_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dbe2cd7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 -653.0150756835938 0.35074582695961 -980.25\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=5e-3)\n",
    "\n",
    "    #训练N局\n",
    "    for epoch in range(100):\n",
    "\n",
    "        #一个epoch最少玩N步\n",
    "        steps = 0\n",
    "        while steps < 200:\n",
    "\n",
    "            #玩一局游戏,得到数据\n",
    "            state, action, reward, _ = play()\n",
    "            steps += len(state)\n",
    "\n",
    "            #计算当前state的价值,其实就是Q(state,action),这里是用蒙特卡洛法估计的\n",
    "            value = []\n",
    "            for i in range(len(reward)):\n",
    "                s = 0\n",
    "                for j in range(i, len(reward)):\n",
    "                    s += reward[j] * 0.99**(j - i)\n",
    "                value.append(s)\n",
    "            value = torch.FloatTensor(value).reshape(-1, 1)\n",
    "\n",
    "            #重新计算动作的概率\n",
    "            prob = model(state).gather(dim=1, index=action)\n",
    "\n",
    "            #求Q最大的导函数 -> partial value / partial action\n",
    "            loss = (prob + 1e-8).log() * value\n",
    "            for i in range(len(loss)):\n",
    "                loss[i] = loss[i] * 0.99**i\n",
    "            loss = -loss.mean()\n",
    "\n",
    "            #计算动作的熵,越大越好\n",
    "            entropy = prob * (prob + 1e-8).log()\n",
    "            entropy = -entropy.mean()\n",
    "            loss -= entropy * 5\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, loss.item(), entropy.item(), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a101d0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAURElEQVR4nO3db2xT97kH8O9xEjuE5DgkLPYiYhGp3NJc/mwECF5fDLUZaRdNY+RK3YS6FCGqMgeVZkJargoIVCmI6V7WbhBeTIO+6bKbSmwiou2NQgnaMKSkzV0IEDGpvYkKtvlzc+wEYic+z33BclpDoPnrX0y+H+lI9e/32H7OKeer43N8HE1EBERECthUN0BEcxcDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlFEWQIcPH8bixYuRmZmJsrIytLe3q2qFiBRREkB/+tOfUFtbi7179+LTTz/FypUrUVFRgVAopKIdIlJEU3EzallZGdasWYPf/e53AADTNFFUVIQdO3bgV7/6VbLbISJF0pP9hrFYDB0dHairq7PGbDYbysvL4ff7x3xONBpFNBq1HpumiTt37iA/Px+aps14z0Q0MSKCSCSCwsJC2GyP/qCV9AC6desW4vE4XC5XwrjL5cLVq1fHfE59fT327duXjPaIaBr19fVh0aJFj5xPegBNRl1dHWpra63HhmHA4/Ggr68Puq4r7IyIxhIOh1FUVIScnJzH1iU9gBYuXIi0tDQEg8GE8WAwCLfbPeZzHA4HHA7HQ+O6rjOAiGaxbzpFkvSrYHa7HaWlpWhtbbXGTNNEa2srvF5vstshIoWUfASrra1FdXU1Vq9ejbVr1+I3v/kNBgcHsWXLFhXtEJEiSgLopZdews2bN7Fnzx4EAgF85zvfwYcffvjQiWkierIp+R7QVIXDYTidThiGwXNARLPQePdR3gtGRMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZSYcQGfPnsWPfvQjFBYWQtM0/PnPf06YFxHs2bMH3/72tzFv3jyUl5fj2rVrCTV37tzB5s2boes6cnNzsXXrVgwMDExpRYgo9Uw4gAYHB7Fy5UocPnx4zPmDBw/inXfewdGjR3HhwgXMnz8fFRUVGBoasmo2b96M7u5utLS0oLm5GWfPnsWrr746+bUgotQkUwBATpw4YT02TVPcbrf8+te/tsb6+/vF4XDIH//4RxERuXz5sgCQTz75xKr54IMPRNM0+fLLL8f1voZhCAAxDGMq7RPRDBnvPjqt54A+//xzBAIBlJeXW2NOpxNlZWXw+/0AAL/fj9zcXKxevdqqKS8vh81mw4ULF8Z83Wg0inA4nLAQUeqb1gAKBAIAAJfLlTDucrmsuUAggIKCgoT59PR05OXlWTUPqq+vh9PptJaioqLpbJuIFEmJq2B1dXUwDMNa+vr6VLdERNNgWgPI7XYDAILBYMJ4MBi05txuN0KhUML8yMgI7ty5Y9U8yOFwQNf1hIWIUt+0BlBxcTHcbjdaW1utsXA4jAsXLsDr9QIAvF4v+vv70dHRYdWcPn0apmmirKxsOtsholkufaJPGBgYwD/+8Q/r8eeff47Ozk7k5eXB4/Fg586deOutt7BkyRIUFxdj9+7dKCwsxMaNGwEAzzzzDF544QVs27YNR48exfDwMGpqavDTn/4UhYWF07ZiRJQCJnp57eOPPxYADy3V1dUicv9S/O7du8XlconD4ZDnn39eenp6El7j9u3b8rOf/Uyys7NF13XZsmWLRCKRab/ER0RqjHcf1UREFObfpITDYTidThiGwfNBRLPQePfRlLgKRkRPJgYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpMyE/ywPEU2dmHH093bBHI5aY9nup2DPzoOmaQo7Sy4GEJEC5sgwev/WiNjAHWuseP0ryP8Xr8Kuko8fwYhmCXMkprqFpGMAEc0SDCAiUoYBRETJoWlIs89LGIrdNRQ1ow4DiEgBzZYGh16QMBY1Qoq6UYcBRKSApmmwpWeobkO5CQVQfX091qxZg5ycHBQUFGDjxo3o6elJqBkaGoLP50N+fj6ys7NRVVWFYDCYUNPb24vKykpkZWWhoKAAu3btwsjIyNTXhihlaLClMYAmFEBtbW3w+Xw4f/48WlpaMDw8jA0bNmBwcNCqeeONN3Dy5Ek0NTWhra0N169fx6ZNm6z5eDyOyspKxGIxnDt3Du+++y6OHz+OPXv2TN9aEc12mgZbul11F+rJFIRCIQEgbW1tIiLS398vGRkZ0tTUZNVcuXJFAIjf7xcRkVOnTonNZpNAIGDVNDQ0iK7rEo1Gx/W+hmEIADEMYyrtEyljmqb0nvsvaT+6zVqunvxPMU1TdWvTYrz76JTOARnG/bP2eXl5AICOjg4MDw+jvLzcqlm6dCk8Hg/8fj8AwO/3Y/ny5XC5XFZNRUUFwuEwuru7x3yfaDSKcDicsBClMk3TgAduuRBzBBBTUUdqTDqATNPEzp078eyzz2LZsmUAgEAgALvdjtzc3IRal8uFQCBg1Xw9fEbnR+fGUl9fD6fTaS1FRUWTbZto1jLjcYgZV91GUk06gHw+Hy5duoTGxsbp7GdMdXV1MAzDWvr6+mb8PYmSTcwRyBw7AprUzag1NTVobm7G2bNnsWjRImvc7XYjFouhv78/4SgoGAzC7XZbNe3t7QmvN3qVbLTmQQ6HAw6HYzKtEqUMiY9AzLkVQBM6AhIR1NTU4MSJEzh9+jSKi4sT5ktLS5GRkYHW1lZrrKenB729vfB679/l6/V60dXVhVDoqy9dtbS0QNd1lJSUTGVdiFKKPScfwFfngYbvhRGP3VXXkAITOgLy+Xx477338Je//AU5OTnWORun04l58+bB6XRi69atqK2tRV5eHnRdx44dO+D1erFu3ToAwIYNG1BSUoKXX34ZBw8eRCAQwJtvvgmfz8ejHJpT7Fm5909EiwAA4sNRmPG59X24CQVQQ0MDAGD9+vUJ48eOHcMrr7wCADh06BBsNhuqqqoQjUZRUVGBI0eOWLVpaWlobm7G9u3b4fV6MX/+fFRXV2P//v1TWxOiFMPvAQGayD/jN4WEw2E4nU4YhgFd11W3QzQpkRvXcPXkf1iX3rW0DPzrv+3GvNyxz4WmkvHuo7wXjEgRW3rGnPr51bEwgIgU0WwPngERfhGRiJLlgaMfuf9b0XMJA4ho1pA596uIDCCiWYQBRETJoSHhhlQRgTkSfXT9E4gBRKRIRmY2MrJyvxoQE/fuXFfWjwoMICJFtLSMh36W1eTd8ESUDJrNBs2WproNpRhARIpoWhpsD30XaG5hABEpotnSxjwCSsG7oyaNAUSkiqYBWuIuyF9EJCJl7n8PiEdARKSAOTI8l/KHAUQ0m0icR0BElCRp9syEx8NDAzwJTUTJMW9BYcLjqBGaUyeiGUBECs31n2VlABEpxAAiImUYQESkzIM3o841DCAiRTRNg/bgN6HF5EloIlJDTBMyh/44IQOIaDYRE6Y5dwJobv8WANEMExEMDg5iZGTsULl7N/FvwZvxEYT7/w8ZI2MfG2iahpycHNhsT8axAwOIaIZt2bIFfr9/zLmVxXn495fWICP9/s9yDN2NYPNLm3D5f2+PWZ+Tk4PW1lYUFhaOOZ9qGEBEM+zWrVv48ssvx5ybbxuCEc3C9SEvYmYmFs/rwoDx34+sdzqdiMefnJPUEzqOa2howIoVK6DrOnRdh9frxQcffGDNDw0NwefzIT8/H9nZ2aiqqkIwGEx4jd7eXlRWViIrKwsFBQXYtWvXIw9PiZ50d2M2fBZej+vRJbg17MH/RJ5DZCRfdVtJM6EAWrRoEQ4cOICOjg5cvHgRzz33HH784x+ju7sbAPDGG2/g5MmTaGpqQltbG65fv45NmzZZz4/H46isrEQsFsO5c+fw7rvv4vjx49izZ8/0rhVRihgaBgZGcjD6V1JjZiaikqW2qWSSKVqwYIH8/ve/l/7+fsnIyJCmpiZr7sqVKwJA/H6/iIicOnVKbDabBAIBq6ahoUF0XZdoNDru9zQMQwCIYRhTbZ9oRpmmKevXrxfc/42Nh5Y8PVsO7T0i+986L/veape3DzTLqmeefmS90+mU3t5e1av1jca7j076HFA8HkdTUxMGBwfh9XrR0dGB4eFhlJeXWzVLly6Fx+OB3+/HunXr4Pf7sXz5crhcLqumoqIC27dvR3d3N7773e9OqIerV68iOzt7sqtANOPkn1fBHmXw3j38re0owngKMXHAZf8CwZuP/ttg8Xgc165dQyQSmYl2p83AwMC46iYcQF1dXfB6vRgaGkJ2djZOnDiBkpISdHZ2wm63Izc3N6He5XIhEAgAAAKBQEL4jM6Pzj1KNBpFNPrVX4wMh8MAAMMweP6IZr3H/RuNDsfx/pm/A/j7uF5LRBCJRJCZmfnNxQo9LnS/bsIB9PTTT6OzsxOGYeD9999HdXU12traJtzgRNTX12Pfvn0PjZeVlUHX9Rl9b6KpEBE4nc5pe7309HSsWrUKRUVF0/aaM2H0IOGbTPjbTHa7HU899RRKS0tRX1+PlStX4u2334bb7UYsFkN/f39CfTAYhNvtBgC43e6HroqNPh6tGUtdXR0Mw7CWvr6+ibZNRLPQlL9OaZomotEoSktLkZGRgdbWVmuup6cHvb298Hq9AACv14uuri6EQiGrpqWlBbquo6Sk5JHv4XA4rEv/owsRpb4JfQSrq6vDiy++CI/Hg0gkgvfeew9nzpzBRx99BKfTia1bt6K2thZ5eXnQdR07duyA1+vFunXrAAAbNmxASUkJXn75ZRw8eBCBQABvvvkmfD4fHA7HjKwgEc1eEwqgUCiEn//857hx4wacTidWrFiBjz76CD/4wQ8AAIcOHYLNZkNVVRWi0SgqKipw5MgR6/lpaWlobm7G9u3b4fV6MX/+fFRXV2P//v3Tu1ZEs8j8+fOn7aj9SboPDAA0kdT7Cf5wOAyn0wnDMPhxjGY1EcHNmzcTruJOhaZpcLvdSE+f3XdRjXcfnd1rQZTiNE1DQUGB6jZmrSfnWI6IUg4DiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhImXTVDUyGiAAAwuGw4k6IaCyj++bovvooKRlAt2/fBgAUFRUp7oSIHicSicDpdD5yPiUDKC8vDwDQ29v72JWjROFwGEVFRejr64Ou66rbSQncZpMjIohEIigsLHxsXUoGkM12/9SV0+nkP4pJ0HWd222CuM0mbjwHBzwJTUTKMICISJmUDCCHw4G9e/fC4XCobiWlcLtNHLfZzNLkm66TERHNkJQ8AiKiJwMDiIiUYQARkTIMICJSJiUD6PDhw1i8eDEyMzNRVlaG9vZ21S0pU19fjzVr1iAnJwcFBQXYuHEjenp6EmqGhobg8/mQn5+P7OxsVFVVIRgMJtT09vaisrISWVlZKCgowK5duzAyMpLMVVHmwIED0DQNO3futMa4zZJEUkxjY6PY7Xb5wx/+IN3d3bJt2zbJzc2VYDCoujUlKioq5NixY3Lp0iXp7OyUH/7wh+LxeGRgYMCqee2116SoqEhaW1vl4sWLsm7dOvne975nzY+MjMiyZcukvLxcPvvsMzl16pQsXLhQ6urqVKxSUrW3t8vixYtlxYoV8vrrr1vj3GbJkXIBtHbtWvH5fNbjeDwuhYWFUl9fr7Cr2SMUCgkAaWtrExGR/v5+ycjIkKamJqvmypUrAkD8fr+IiJw6dUpsNpsEAgGrpqGhQXRdl2g0mtwVSKJIJCJLliyRlpYW+f73v28FELdZ8qTUR7BYLIaOjg6Ul5dbYzabDeXl5fD7/Qo7mz0MwwDw1Q27HR0dGB4eTthmS5cuhcfjsbaZ3+/H8uXL4XK5rJqKigqEw2F0d3cnsfvk8vl8qKysTNg2ALdZMqXUzai3bt1CPB5P+J8OAC6XC1evXlXU1exhmiZ27tyJZ599FsuWLQMABAIB2O125ObmJtS6XC4EAgGrZqxtOjr3JGpsbMSnn36KTz755KE5brPkSakAosfz+Xy4dOkS/vrXv6puZVbr6+vD66+/jpaWFmRmZqpuZ05LqY9gCxcuRFpa2kNXI4LBINxut6KuZoeamho0Nzfj448/xqJFi6xxt9uNWCyG/v7+hPqvbzO32z3mNh2de9J0dHQgFAph1apVSE9PR3p6Otra2vDOO+8gPT0dLpeL2yxJUiqA7HY7SktL0draao2ZponW1lZ4vV6FnakjIqipqcGJEydw+vRpFBcXJ8yXlpYiIyMjYZv19PSgt7fX2mZerxddXV0IhUJWTUtLC3RdR0lJSXJWJImef/55dHV1obOz01pWr16NzZs3W//NbZYkqs+CT1RjY6M4HA45fvy4XL58WV599VXJzc1NuBoxl2zfvl2cTqecOXNGbty4YS137961al577TXxeDxy+vRpuXjxoni9XvF6vdb86CXlDRs2SGdnp3z44YfyrW99a05dUv76VTARbrNkSbkAEhH57W9/Kx6PR+x2u6xdu1bOnz+vuiVlAIy5HDt2zKq5d++e/OIXv5AFCxZIVlaW/OQnP5EbN24kvM4XX3whL774osybN08WLlwov/zlL2V4eDjJa6POgwHEbZYc/DkOIlImpc4BEdGThQFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEp8//qMNiVA32qHgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
